/**
 * main.js
 * 2020年1月6日 农历腊月十二
 */
//const repeat = require('./repeater');
const db = require('../tools/mongodb');
const db_config = require('./mongodb.config');
const agent = require('./agent');
const delay = require('../tools/delay');
const format = require('../tools/format');
format();
const commentsFn = require('./comments');
const waitMillis = 100;
const randomMillis = function () {
    return Math.random() * 100;
}
const logger = require('../tools/logger');
const newComponentLimit= 100;

let app = async () => {
    const summary = db_config.jd.summary;
    const replies = db_config.jd.replies;
    const images = db_config.jd.images;
    const videos = db_config.jd.videos;
    const comments = db_config.jd.comments;
    const log = db_config.jd.log;
    const trace = db_config.jd.trace;
    //const commentIdTrack = db_config.jd.commentIdTrack;
    let logId = Number(new Date());
    let updateList = [];//记录有变更的skuId用于最后log
    let startTime = new Date(logId).format('yyyy-MM-dd HH:mm:ss');
    try {
        let _idArray = [];//存储_id 减少内存占用————2020年3月20日
        await db.find(summary, { 'disabled': { $ne: '1' } }).then(async r => {
            _idArray =  r.map(item=>{
                return item._id;
            });
        }).catch(error=>{throw error;});
        console.log(`本次任务需处理${_idArray.length}个商品`);
        db.insertOne(log, {
            'logId':logId,
            start: startTime,
            content: `本次任务需处理${_idArray.length}个商品`,
            operation: "crawl"
        });
        for(let _id of _idArray){
            let smy = {};
            await db.find(summary,{'_id':_id}).then(r=>{
                smy = r[0];
            }).catch(error=>{throw error;});
            await db.deleteMany(trace, {});
                //smy update字段指只要完成循环即更新；
                let update = false;//summary是否有变更
                let change = false;//评论数（score1~5）是否有改变————新的更新时间规则
                delete smy._id;//避免重复id
                await db.updateMany(summary, { skuId: smy.skuId }, { skuId: smy.skuId.toString() });//设置skuId 为字符串
                smy.skuId = smy.skuId.toString();
                let logTrack = [];//记录所有的操作;
                try {
                    //console.clear();//不保留太多console
                    console.log(`${new Date().format('yyyy-MM-dd HH:mm:ss')}:${smy.skuId}`);
                    logTrack.push({ time: new Date(), skuId: smy.skuId, callback: smy.callback, memo: `开始获取` });
                    //获取并更新summary
                    try {
                        await agent.getSummary(smy.skuId)
                            .then(async r => {
                                await db.updateMany(summary, { skuId: smy.skuId.toString() }, r);
                            })
                            .catch(error => {
                                throw error;
                            });
                    }
                    catch (error) {
                        //console.error(error);
                        continue;
                    }
                    let smy_new = {};
                    //获取更新后的summary数据
                    await db.find(summary, { skuId: smy.skuId }).then(r => {
                        smy_new = r[0];
                        smy_new.skuId = smy.skuId.toString();
                        delete smy._id;//避免重复id
                    }).catch(error => { throw error; });
                    //获取首页数据(各类评论数量)
                    await delay(randomMillis());
                    let firstPage = {};
                    try {
                        await agent.getComments(smy_new, 0).then(async r => {
                            firstPage = commentsFn.firstPage(r);
                            //await db.updateMany(summary, { skuId: smy_new.skuId }, firstPage);
                        }).catch(error => {
                            throw error;
                        });
                        logTrack.push({ time: new Date(), memo: `获取首页数据` });
                    }
                    catch (error) {
                        logger.error(trace, { skuId: smy_new.skuId, error: error, memo: `首页信息获取失败` });//记录错误信息
                        console.error(error);
                        //写入日志
                        logTrack.push({ time: new Date(), memo: `首页信息获取失败，切换下一个商品` });
                        continue;//切换下一个品番
                    }
                    //以首页获取到的评论数量作为判断条件,获取各评分评论
                    let compare = commentsFn.scoreCompare(smy_new, firstPage);//Map类型
                    for (let pair of compare) {
                        let score = pair[0]; //ajax参数
                        let increment = pair[1];//差值
                        if (increment == 0) continue;
                        //summary评论数commentCount为0的品号，只取前newComponentLimit条记录
                        if(Number(smy_new.commentCount)==0){
                            increment = increment<newComponentLimit?increment:newComponentLimit;
                        }
                        let gain = 0, page = 0;
                        let emptyPage = false;
                        let gotMaxId = false;
                        while (gain < increment) {
                            try {
                                await agent.getComments(smy_new, page, score).then(async r => {
                                    logTrack.push({ time: new Date(), memo: `分类${score}第${page}页，获取到${r.comments.length}条记录` });
                                    await db.insertOne(trace, { memo: `分类${score}第${page}页，获取到${r.comments ? r.comments.length : 0}条记录`, time: new Date() });
                                    page++;
                                    if (r.comments.length == 0) {
                                        emptyPage = true;
                                        throw `score${score}page${page}为空!`;
                                    }
                                    let arrays = {
                                        body: [],
                                        replies: [],
                                        images: [],
                                        videos: [],
                                    };
                                    for (let c of r.comments) {
                                        await db.insertOne(trace, { comment: c, time: new Date() });
                                        let _comment = commentsFn.convertCommentBody(c);
                                        _comment.skuId = smy_new.skuId;
                                        //拿到之前的最大id则跳出
                                        if (_comment.id == smy_new.comments_maxId) {
                                            gotMaxId = true;
                                            break;
                                        }
                                        arrays.body.push(_comment);
                                        let _replies = commentsFn.convertCommentReplies(c);
                                        arrays.replies = [...arrays.replies, ..._replies];
                                        let _images = commentsFn.convertCommentImages(c);
                                        arrays.images = [...arrays.images, ..._images];
                                        let _videos = commentsFn.convertCommentVideos(c);
                                        arrays.videos = [...arrays.videos, ..._videos];
                                    }
                                    if (arrays.body.length > 0) {
                                        await db.insertMany(comments, arrays.body).then(async r => {
                                            // await db.insertOne(trace, { data: arrays.body, info: r, time: new Date() });
                                        }).catch(error => {
                                            throw { ...error, ...arrays }
                                        });
                                        if (arrays.replies.length > 0) {
                                            await db.insertMany(replies, arrays.replies).then(async r => {
                                                // await db.insertOne(trace, { data: [replies, arrays.replies], info: r, time: new Date() });
                                            }).catch(error => {
                                                throw { ...error, ...arrays };
                                            });
                                        }
                                        if (arrays.images.length > 0) {
                                            await db.insertMany(images, arrays.images).then(async r => {
                                                // await db.insertOne(trace, { data: [images, arrays.images], info: r, time: new Date() });
                                            }).catch(error => {
                                                throw { ...error, ...arrays };
                                            });
                                        }
                                        if (arrays.videos.length > 0) {
                                            await db.insertMany(videos, arrays.videos).then(async r => {
                                                // await db.insertOne(trace, { data: [videos, arrays.videos], info: r, time: new Date() });
                                            }).catch(error => {
                                                throw { ...error, ...arrays };
                                            });
                                        }
                                    }
                                    gain += arrays.body.length;//跳出1，数量足够
                                }).catch(error => {
                                    throw error;
                                });
                                if (gotMaxId) {
                                    //跳出2，已检索到旧记录
                                    logTrack.push({ time: new Date(), memo: `访问到已有记录，跳出循环!` });
                                    console.log(`访问到已有记录，跳出循环!`);
                                    break;
                                }
                            }
                            catch (error) {
                                //db记录错误
                                if (emptyPage) {
                                    logTrack.push({ time: new Date(), memo: `获取到空页，跳出循环!` });
                                    break;
                                }
                                else{
                                    console.error(error);
                                }
                                logger.error(trace, { skuId: smy_new.skuId, error: error });
                                continue;
                            }
                            await delay(randomMillis());//分页延时
                        }
                        update = true;
                        change = true;
                    }
                    /**更新afterCount:追评 score = 5
                       * 更新replies:客服回应 score 无 
                       * 更新images:晒图 score = 4
                       * 更新videos:视频晒单 score = 7
                      */
                    let compare_A = firstPage.afterCount - smy_new.afterCount;
                    if (compare_A > 0) {
                        //summary评论数commentCount为0的品号，只取前newComponentLimit条记录
                        if(Number(smy_new.commentCount)==0){
                            compare_A = compare_A<newComponentLimit?compare_A:newComponentLimit;
                        }
                        logTrack.push({ time: new Date(), memo: `开始更新追评信息` });
                        let gain = 0;
                        let page = 0;
                        let emptyPage = false;
                        while (gain < compare_A) {
                            try {
                                await agent.getComments(smy_new, page, 5).then(async r => {
                                    logTrack.push({ time: new Date(), memo: `分类5(追评)第${page}页，获取到${r.comments ? r.comments.length : 0}条记录` });
                                    await db.insertOne(trace, { memo: `分类5(追评)第${page}页，获取到${r.comments ? r.comments.length : 0}条记录`, time: new Date() });
                                    page++;
                                    if (r.comments.length == 0) {
                                        emptyPage = true;
                                        throw `分类5(追评)page${page}为空，结束循环!`;
                                    }
                                    let update = [], insert = [];//分组记录
                                    for (let c of r.comments) {
                                        //判断insert/update
                                        let after = commentsFn.convertAfterComment(c);
                                        after.body.skuId = smy_new.skuId;
                                        await db.find(comments, { id: after.afterUserComment_commentId }).then(async r => {
                                            if (r.length > 0) {
                                                update.push(after);
                                            }
                                            else {
                                                after.body = { ...after.body, ...commentsFn.convertCommentBody(c) };
                                                insert.push(after);
                                            }
                                        });
                                    }
                                    let arrays = {
                                        replies: [],
                                        images: [],
                                        videos: []
                                    };
                                    if (update.length > 0) {
                                        //分4组insert/update
                                        for (let u of update) {
                                            await db.updateMany(comments, { id: u.body.afterUserComment_commentId }, u.body).then(async r => {
                                                // await db.insertOne(trace, { data: u.body, info: r, time: new Date() });
                                            }).catch(error => {
                                                throw { ...error, ...u }
                                            });
                                            arrays.replies = [...arrays.replies, ...u.replies];
                                            arrays.images = [...arrays.images, ...u.images];
                                            arrays.videos = [...arrays.videos, ...u.videos];
                                        }
                                    }
                                    if (insert.length > 0) {
                                        await db.insertMany(comments, insert.map((item) => {
                                            delete item.body._id;
                                            return item.body;
                                        })).then(async r => {
                                            // await db.insertOne(trace, { data: _insert, info: r, time: new Date() });
                                        }).catch(error => {
                                            throw { ...error, ...i }
                                        });
                                        for (let i of insert) {
                                            arrays.replies = [...arrays.replies, ...i.replies];
                                            arrays.images = [...arrays.images, ...i.images];
                                            arrays.videos = [...arrays.videos, ...i.videos];
                                        }
                                    }
                                    if (arrays.replies.length > 0) {
                                        await db.insertMany(replies, arrays.replies).then(async r => {
                                            // await db.insertOne(trace, { data: [replies, arrays.replies], info: r, time: new Date() });
                                        }).catch(error => {
                                            throw { ...error, ...arrays };
                                        });
                                    }
                                    if (arrays.images.length > 0) {
                                        await db.insertMany(images, arrays.images).then(async r => {
                                            // await db.insertOne(trace, { data: [images, arrays.images], info: r, time: new Date() });
                                        }).catch(error => {
                                            throw { ...error, ...arrays };
                                        });
                                    }
                                    if (arrays.videos.length > 0) {
                                        await db.insertMany(videos, arrays.videos).then(async r => {
                                            // await db.insertOne(trace, { data: [videos, arrays.videos], info: r, time: new Date() });
                                        }).catch(error => {
                                            throw { ...error, ...arrays };
                                        });
                                    }
                                    gain += (update.length + insert.length);
                                }).catch(error => {
                                    throw error;
                                });
                            }
                            catch (error) {
                                if (emptyPage) {
                                    logTrack.push({ time: new Date(), memo: `获取到空页，跳出循环!` });
                                    break;
                                }
                                else{
                                    console.error(error);
                                }
                                continue;
                            }
                            await delay(randomMillis());//分页延时
                        }
                        update = true;
                    }
                    /**2020年1月19日追加 视频 && 图片评论 (基本评论内信息不全) */
                    let compare_I = firstPage.imageListCount - smy_new.imageListCount;
                    if (compare_I > 0) {
                        console.log(`开始更新晒图信息`);
                        logTrack.push({ time: new Date(), memo: `开始更新晒图信息` });
                        if(Number(smy_new.commentCount)==0){
                            compare_I = compare_I<newComponentLimit?compare_I:newComponentLimit;
                        }
                        let gain = 0;
                        let page = 0;
                        let emptyPage = false;
                        while (gain < compare_I) {
                            try {
                                await agent.getComments(smy_new, page, 4).then(async r => {
                                    logTrack.push({ time: new Date(), memo: `分类4(晒图)第${page}页，获取到${r.comments.length}条记录` });
                                    await db.insertOne(trace, { memo: `分类4(晒图)第${page}页，获取到${r.comments ? r.comments.length : 0}条记录`, time: new Date() });
                                    page++;
                                    if (r.comments.length == 0) {
                                        emptyPage = true;
                                        throw `score4(晒图)page${page}为空!`;
                                    }
                                    let arrays = {
                                        body: [],
                                        replies: [],
                                        images: [],
                                        videos: [],
                                    };

                                    for (let c of r.comments) {
                                        //逐个判断是否写入
                                        let found = false;
                                        await db.find(comments, { 'id': c.id }).then(r => {
                                            if (r.length > 0) {
                                                found = true;
                                            }
                                        });
                                        if (found === false) {
                                            await db.insertOne(trace, { comment: c, time: new Date() });
                                            let _comment = commentsFn.convertCommentBody(c);
                                            _comment.skuId = smy_new.skuId;
                                            //补充追评 2020年1月19日
                                            let after = {};
                                            let _replies = commentsFn.convertCommentReplies(c);
                                            let _images = commentsFn.convertCommentImages(c);
                                            let _videos = commentsFn.convertCommentVideos(c);
                                            if (c.afterUserComment && c.afterUserComment.id) {
                                                let after = commentsFn.convertAfterComment(c);
                                                // 'body': after,
                                                // 'replies': replies,
                                                // 'images': images,
                                                // 'videos': videos//after format
                                                _comment = { ..._comment, ...after.body };
                                                _images = [..._images, ...after.images];
                                                _videos = [..._videos, ...after.videos];
                                                _replies = [..._replies, ...after.replies];
                                            }
                                            arrays.body.push(_comment);
                                            arrays.replies = [...arrays.replies, ..._replies];
                                            arrays.images = [...arrays.images, ..._images];
                                            arrays.videos = [...arrays.videos, ..._videos];
                                        }
                                    }
                                    if (arrays.body.length > 0) {
                                        await db.insertMany(comments, arrays.body).then(async r => {
                                            await db.insertOne(trace, { data: arrays.body, info: r, time: new Date() });
                                        }).catch(error => {
                                            throw { ...error, ...arrays }
                                        });
                                        if (arrays.replies.length > 0) {
                                            await db.insertMany(replies, arrays.replies).then(async r => {
                                                await db.insertOne(trace, { data: [replies, arrays.replies], info: r, time: new Date() });
                                            }).catch(error => {
                                                throw { ...error, ...arrays };
                                            });
                                        }
                                        if (arrays.images.length > 0) {
                                            await db.insertMany(images, arrays.images).then(async r => {
                                                await db.insertOne(trace, { data: [images, arrays.images], info: r, time: new Date() });
                                            }).catch(error => {
                                                throw { ...error, ...arrays };
                                            });
                                        }
                                        if (arrays.videos.length > 0) {
                                            await db.insertMany(videos, arrays.videos).then(async r => {
                                                await db.insertOne(trace, { data: [videos, arrays.videos], info: r, time: new Date() });
                                            }).catch(error => {
                                                throw { ...error, ...arrays };
                                            });
                                        }
                                    }
                                    gain += r.comments.length;//并非全部写入，总数符合即可
                                }).catch(error => {
                                    throw error;
                                });
                            }
                            catch (error) {
                                
                                if (emptyPage) {
                                    logTrack.push({ time: new Date(), memo: `获取到空页，跳出循环!` });
                                    break;
                                }
                                else{
                                    console.error(error);
                                }
                                continue;
                            }
                            await delay(randomMillis());//分页延时
                        }
                        update = true;
                    }


                    let compare_V = firstPage.videoCount - smy_new.videoCount;
                    if (compare_V > 0) {
                        logTrack.push({ time: new Date(), memo: `开始更新视频晒单信息` });
                        console.log(`开始更新视频晒单信息`)
                        if(Number(smy_new.commentCount)==0){
                            compare_V = compare_V<newComponentLimit?compare_V:newComponentLimit;
                        }
                        let gain = 0;
                        let page = 0;
                        let emptyPage = false;
                        while (gain < compare_V) {
                            try {
                                await agent.getComments(smy_new, page, 7).then(async r => {
                                    logTrack.push({ time: new Date(), memo: `分类7(视频晒单)第${page}页，获取到${r.comments.length}条记录` });
                                    await db.insertOne(trace, { memo: `分类7(视频晒单)第${page}页，获取到${r.comments ? r.comments.length : 0}条记录`, time: new Date() });
                                    page++;
                                    if (r.comments.length == 0) {
                                        emptyPage = true;
                                        throw `score7(视频晒单)page${page}为空!`;
                                    }
                                    let arrays = {
                                        body: [],
                                        replies: [],
                                        images: [],
                                        videos: [],
                                    };

                                    for (let c of r.comments) {

                                        //逐个判断是否写入
                                        let found = false;
                                        await db.find(comments, { id: c.id }).then(r => {
                                            if (r.length > 0) {
                                                found = true;
                                            }
                                        });
                                        if (found === false) {
                                            await db.insertOne(trace, { comment: c, time: new Date() });
                                            let _comment = commentsFn.convertCommentBody(c);
                                            _comment.skuId = smy_new.skuId;
                                            //补充追评 2020年1月19日
                                            let after = {};
                                            let _replies = commentsFn.convertCommentReplies(c);
                                            let _images = commentsFn.convertCommentImages(c);
                                            let _videos = commentsFn.convertCommentVideos(c);
                                            if (c.afterUserComment && c.afterUserComment.id) {
                                                let after = commentsFn.convertAfterComment(c);
                                                // 'body': after,
                                                // 'replies': replies,
                                                // 'images': images,
                                                // 'videos': videos//after format
                                                _comment = { ..._comment, ...after.body };
                                                _comment.skuId = smy_new.skuId;
                                                _images = [..._images, ...after.images];
                                                _videos = [..._videos, ...after.videos];
                                                _replies = [..._replies, ...after.replies];
                                            }
                                            arrays.body.push(_comment);
                                            arrays.replies = [...arrays.replies, ..._replies];
                                            arrays.images = [...arrays.images, ..._images];
                                            arrays.videos = [...arrays.videos, ..._videos];
                                        }
                                    }
                                    if (arrays.body.length > 0) {
                                        // await db.insertMany(commentIdTrack, arrays.body.map(item => { return {id:item.id}; }));
                                        await db.insertMany(comments, arrays.body).then(async r => {
                                            await db.insertOne(trace, { data: arrays.body, info: r, time: new Date() });
                                        }).catch(error => {
                                            throw { ...error, ...arrays }
                                        });
                                        if (arrays.replies.length > 0) {
                                            await db.insertMany(replies, arrays.replies).then(async r => {
                                                await db.insertOne(trace, { data: [replies, arrays.replies], info: r, time: new Date() });
                                            }).catch(error => {
                                                throw { ...error, ...arrays };
                                            });
                                        }
                                        if (arrays.images.length > 0) {
                                            await db.insertMany(images, arrays.images).then(async r => {
                                                await db.insertOne(trace, { data: [images, arrays.images], info: r, time: new Date() });
                                            }).catch(error => {
                                                throw { ...error, ...arrays };
                                            });
                                        }
                                        if (arrays.videos.length > 0) {
                                            await db.insertMany(videos, arrays.videos).then(async r => {
                                                await db.insertOne(trace, { data: [videos, arrays.videos], info: r, time: new Date() });
                                            }).catch(error => {
                                                throw { ...error, ...arrays };
                                            });
                                        }
                                    }
                                    gain += r.comments.length;//并非全部写入，总数符合即可
                                }).catch(error => {
                                    throw error;
                                });
                            }
                            catch (error) {
                                if (emptyPage) {
                                    logTrack.push({ time: new Date(), memo: `获取到空页，跳出循环!` });
                                    break;
                                }
                                else{
                                    console.error(error);
                                }
                                continue;
                            }
                            await delay(randomMillis());//分页延时
                        }
                        update = true;
                    }

                    await delay(randomMillis());//延时减轻写入负载
                    let summary_update = {
                        ...firstPage, ...{ update: new Date().format("yyyy-MM-dd HH:mm:ss")}
                    };
                    if (change == true) {
                        summary_update = {
                            ...summary_update, ...{ changeTime: new Date().format("yyyy-MM-dd HH:mm:ss") }
                        };
                    }
                    await db.updateMany(summary, { skuId: smy.skuId }, summary_update);
                    logTrack.push({ time: new Date(), memo: `更新summary!` });
                    if (update) {
                        updateList.push(smy.skuId);
                    }

                }
                catch (error) {
                   // console.error(error);
                    continue;
                }
                finally {
                    logger.trace(trace, { skuId: smy.skuId, logTrack: logTrack });
                }
            //}//for(let smy of r)
        }
        await db.updateMany(log,{'logId':logId}, {
            end: new Date().format('yyyy-MM-dd HH:mm:ss'),
            content: `本次任务共变更${updateList.length}个商品`,
            updateList: updateList
        });
    }
    catch (error) {
        await db.insertOne(db_config.jd.error, { 'logId':logId,error: error,stack :error.stack, time: new Date() });
        console.error(error);
    }
}
module.exports = app;